from datasets import load_dataset,Dataset
from transformers import BertModel, BertTokenizer
import warnings
warnings.filterwarnings("ignore")

model_name = '../bert-base-chinese'
model_path = '../bert-base-chinese'

tokenizer = BertTokenizer.from_pretrained(model_name)
bert = BertModel.from_pretrained(model_path)
import pandas as pd

df = pd.read_csv('../datasets/train.csv')


dataset = Dataset.from_pandas(df)



print(dataset)
print(dataset["text"][0])
#
# pip install datasets transformers sacrebleu sentencepiece
